# 需求爬取梨视频的视频数据
# 原则：线程池处理的是阻塞并且耗时的操作
import json
import random
from multiprocessing.dummy import Pool

import numpy
import requests
from lxml import etree

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.3'}

url = 'https://www.pearvideo.com/popular_8'
page_text = requests.get(url=url, headers=headers).text

tree = etree.HTML(page_text)

li_list = tree.xpath('//*[@id="popularList"]/li')
urls = []
for li in li_list:
    detail_url = 'https://www.pearvideo.com/' + li.xpath('./a/@href')[0]
    name = li.xpath('./div[2]/a/h2/text()')[0] + '.mp4'
    print(detail_url, name)
    # 对详情页发起请求
    detail_page_text = requests.get(url=detail_url, headers=headers).text
    date_tree = etree.HTML(detail_page_text)
    data_id = date_tree.xpath('//*[@id="detailsbd"]/div[1]/div[2]/div/div[1]/div/div[2]/@data-id')[0]
    # 生成mrd
    ran = ['0.']
    for i in range(0, 18):
        ran.append(str(random.randint(0, 9)))
    a = ''.join(ran)

    video_url = 'https://www.pearvideo.com/videoStatus.jsp?contId=' + data_id + '&mrd=' + str(a)
    video_headers = {
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "Connection": "keep-alive",
        "Host": "www.pearvideo.com",
        "Referer": "https://www.pearvideo.com/video_" + data_id,
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin", "Accept": "application/json, text/javascript, */*; q=0.01",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 Edg/85.0.564.63",
        "X-Requested-With": "XMLHttpRequest",
    }
    dic = {
        'name': name,
        'url': video_url,
        'video_headers': video_headers,
        'data_id':data_id
    }
    urls.append(dic)


# 对视频链接发起请求获取视频的二进制数据，然后对数据进行返回
def get_video_data(dic):
    url = dic['url']
    print(dic['name'], '正在下载')
    # 发起ajax请求
    data_dict = requests.get(url=url, headers=dic['video_headers']).content
    if isinstance(data_dict, numpy.ndarray):
        data_json = data_dict.tolist()
    elif isinstance(data_dict, bytes):
        data_json = str(data_dict, encoding='utf-8');
        video_json = json.loads(data_json)
    videoInfo_dic = video_json['videoInfo']
    videos_dic = videoInfo_dic['videos']
    # ajax返回视频请求路径 https://video.pearvideo.com/mp4/third/20201023/1603616503504-10008579-151057-hd.mp4
    srcUrl = videos_dic['srcUrl']
    # 详情页面请求视频路径为https://video.pearvideo.com/mp4/third/20201023/cont-1703247-10008579-151057-hd.mp4
    # 替换时间戳部分为视频id
    time_src = srcUrl[49:62]
    srcUrl_new = srcUrl.replace(time_src, "cont-" + dic['data_id'])
    # 视频请求url
    print(srcUrl_new)
    # video_real_url = str(data_dict['videoInfo'])
    # 请求下载视频
    data_video = requests.get(url=srcUrl_new).content
    with open(dic['name'], 'wb') as fp:
        fp.write(data_video)
        print(dic['name'], '下载成功')


# 使用线程池对视频数据进行请求
pool = Pool(4)
pool.map(get_video_data, urls)

pool.close()
pool.join
